#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          to be called from test21.30
#          Tests if the GrpJob limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test GrpJob limits
proc inc21_30_3 { wait_reason } {

	global salloc scontrol srun acct re_word_str bin_sleep jobmatch number grjobs_num

	log_info "Starting GrpJob limit test"
	array set job_id {}
	set check_num $grjobs_num

	# Raise an error to abort the catch block
	set exception_code [catch {

		set jobmatch 0
			for {set inx 0} {$inx <= $check_num} {incr inx} {
				set test_reason 0
					spawn $salloc --account=$acct -n1 -t1 $srun $bin_sleep 60
					expect {
						-re "Granted job allocation ($number)" {
							set job_id($inx) $expect_out(1,string)
								if {$inx < $check_num} {
									log_debug "Job $inx $job_id($inx) was submitted"
										incr jobmatch
								} else {
									error "Job $inx ($job_id($inx)) wasn't suppose to work but it did, limit was $check_num"
								}
						}
						-re "job ($number) queued and waiting for resources" {
							set job_id($inx) $expect_out(1,string)
								if {$inx >= $check_num} {
									log_debug "Job $inx $job_id($inx) is waiting for resources. This is expected"
										incr jobmatch
								} else {
									set test_reason 1
								}
						}
						-re "error" {
							error "Job allocation should not have failed"
						}
						timeout {
							error "salloc is not responding"
						}
						eof {
							wait
						}
					}

#		Job is waiting, check if the reason is the limit we are testing for
				if {$test_reason != 0 && $job_id($inx) != 0} {
					spawn $scontrol show job $job_id($inx)
						expect {
							-re "Reason=($re_word_str)" {
								if {$expect_out(1,string) ne $wait_reason} {
									error "Job $inx ($job_id($inx)) was suppose to work but it didn't, limit was $check_num"
								} else {
									incr jobmatch
								}
								exp_continue
							}
							timeout {
								error "scontrol is not responding"
							}
							eof {
								wait
							}
						}
				}
			}

		set target_jobs [expr $check_num + 1]
		if {$jobmatch != $target_jobs} {
			error "Not all jobs were not submitted ($jobmatch != $target_jobs)"
		}

		# Checks the job state
		if [info exists job_id($check_num)] {
			subtest [check_job_state $job_id($check_num) PENDING] "Job should be in the PENDING state"
		} else {
			error "Didn't attempt to start enough jobs"
		}

	} message] ; # Store the error message in $message


	# Perform local cleanup
	for {set inx 0} {$inx <= $check_num} {incr inx} {
		if [info exists job_id($inx)] {
			# Cancels the jobs
			cancel_job $job_id($inx)
		}
	}

	# Convert any errors into failures (after cleaning up)
	if {$exception_code == 1} { ; # errors only
		fail "Failure testing GrpJob limits: $message"
	}
}
